package com.solidtrust.crawler.framework;

import java.io.IOException;
import java.net.URL;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.X509TrustManager;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;

import com.solidtrust.crawler.model.PageSource;
import com.solidtrust.dao.CommonDao;
import com.solidtrust.entity.IScrapeEntity;

public abstract class BaseSitemap implements ISitemap{
	private static final Logger log4j = Logger.getLogger(BaseSitemap.class);
	static{
		trustEveryone();
	}
	private static final int TIMEOUT = 1000 * 10;
	protected Date startDate = DateUtils.truncate(new Date(), Calendar.DATE);
	
	protected Map<String, Long> unsold; //数据库里显示上次没有卖出的
	
	//run up memory
	protected Map<String, Long> sold; //数据库里显示上次卖出的
	protected Map<String, Long> existing = new HashMap<>(); //当天网站仍然没有卖出的
	protected Set<String> news = new HashSet<>(); //当天新的
	protected List<IScrapeEntity> newCars = new ArrayList<>();
	protected int totalRequests = 0;
	protected boolean running = false;
	
	@Autowired protected CommonDao commonDao;
	
	
	
	@Override
	public void start() throws Exception{
		if(running == true){
			log4j.info(getSource() + " is running");
			return;
		}
		running = true;
		
		startDate = DateUtils.truncate(new Date(), Calendar.DATE);
		log4j.info("Start to crawl " + getSource() + " on " + startDate);
		
		log4j.info("[" + getSource() + "] start from last ID: " + commonDao.lastId(getEntityClass()));
		
		unsold = commonDao.getUnsoldCarIds(getEntityClass());
		log4j.info("[" + getSource() + "]" + unsold.size() + " unsold cars");
		
//		sold = commonDao.getSoldCarIds(getEntityClass());
//		log4j.info("[" + getSource() + "]" + sold.size() + " sold cars");
		
		
		List<PageSource> cities = null;
		Document doc = null;
		if(getStartUrl() != null){
			doc = parseURL(getStartUrl());
			cities = getCitiesOrProvinces(doc);
		}
		if(cities != null){
			for (PageSource c : cities) {
				log4j.info("Start City " + c);
				crawlByBrandAndModel(null, c);
			}
		}else{
			crawlByBrandAndModel(doc, null);
		}
		
		//save rest cars;
		persist();
		
		log4j.info("[" + getSource() + "]" + "Scrape " + news.size() + " cars");
		
		//touch old records
		log4j.info("[" + getSource() + "]" + "To touch " + existing.size() + " records");
		touchOrRemove();
		
		finish();
		log4j.info("Total requests: " + totalRequests);
		running = false;
	}
	
	protected void crawlByBrandAndModel(Document doc, PageSource c)throws Exception{
		List<PageSource> brands = getBrands(doc, c);
		for (PageSource b : brands) {
			doc = parseURL(b.getUrl());
			List<PageSource> models = getModels(doc, b);
			for (PageSource m : models) {
				nextPage(m, 1, m.getName(), b.getName(), c == null ? null : c.getName());
			}
		}
	}
	
	

	@Override
	public void nextPage(PageSource page, int pageIndex, String model, String brand, String city) throws Exception {
		log4j.info("Start " + page);
		//get data item
		Document doc = parseURL(page.getUrl());
		Elements eles = doc.select(getRowElementSelector());
		if(eles == null || eles.isEmpty()){
			log4j.warn("No result for: " + page.getUrl());
		}else{
			for (Element ele : eles) {
				try {
					IScrapeEntity e = eachItem(ele, model, brand);
					if(e != null){
						if(StringUtils.isNotBlank(brand)){
							e.setBrand(brand);
						}
						if(StringUtils.isNotBlank(model)){
							e.setModel(model);
						}
						if(StringUtils.isNotBlank(city)){
							e.setCity(city);
						}
						toPersist(e);
					}
				} catch (NotSameCityException e) {
					log4j.error(e.getMessage());
					//遇到其他城市的，结束当前城市
					return;
				} catch (Exception e) {
					log4j.error(e.getMessage(), e);
					log4j.info(ele.toString());
				}
			}
			
			PageSource nextPage = getNextPage(doc);
			if(nextPage != null){
				try {
					nextPage.setName(model);
					nextPage(nextPage, pageIndex + 1, model, brand, city);
				} catch (Exception e) {
					log4j.error(e.getMessage(), e);
				}
			}
		}
	}
	
	protected boolean checkExist(String carId, boolean markSold){
		//check unsold
		Long id = unsold.get(carId);
		if(id != null){
			log4j.info(carId + " exists in DB");
			if(markSold == true){
				log4j.info("but marked sold");
			}else{
				existing.put(carId, id);
			}
			return false;
		}
		
		//check sold
		if(sold != null){
			if(sold.containsKey(carId)){
				log4j.info(carId + " has been sold last time");
				return false;
			}
		}else{
			if(commonDao.hasSold(carId, getEntityClass())){
				log4j.info(carId + " has been sold last time");
				return false;
			}
		}
		
		//check new
		if(news.contains(carId)){
			log4j.info(carId + " exists minutes ago");
			return false;
		}
		news.add(carId);
		return true;
	}
	
	protected void toPersist(IScrapeEntity entity){
		entity.setScrapedDate(startDate);
		newCars.add(entity);
		if(newCars.size() == 100){
			persist();
		}
	}
	
	protected void persist(){
		commonDao.saveBatch(newCars);
		newCars.clear();
	}
	
	
	
	protected abstract Class<? extends IScrapeEntity> getEntityClass();
	
	protected abstract String getRowElementSelector();
	
	protected abstract IScrapeEntity eachItem(Element ele, String model, String brand) throws IOException;
	
	protected abstract PageSource getNextPage(Document doc);
	
	/*
	 * 此方法较耗内存
	 */
	protected void touchOrRemove(){
		List<Long> toTouch = new ArrayList<>();
		List<Long> toDelete = new ArrayList<>();
		for (Entry<String, Long> en : unsold.entrySet()) {
			if(existing.containsKey(en.getKey())){
				toTouch.add(en.getValue());
				if(toTouch.size() == 100){
					commonDao.touch(toTouch, startDate, getEntityClass());
					toTouch.clear();
				}
			}else{
				toDelete.add(en.getValue());
				if(toDelete.size() == 100){
					commonDao.markDelete(toDelete, startDate, getEntityClass());
					toDelete.clear();
				}
			}
		}
		commonDao.touch(toTouch, startDate, getEntityClass());
		commonDao.markDelete(toDelete, startDate, getEntityClass());
	}
	
	
	protected Document parseURL(String url) throws Exception{
		log4j.info("Scrape page:" + url);
		totalRequests++;
		Document doc;
		try {
			doc = Jsoup.parse(new URL(url), TIMEOUT);
			return doc;
		} catch (Exception e) {
			log4j.warn(e.getMessage(), e);
		}
		try {
			Thread.sleep(1000 * 60);
		} catch (InterruptedException ie) {}
		return parseURL(url);
	}
	
	public static void trustEveryone() {  
        try {  
            HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {  
                public boolean verify(String hostname, SSLSession session) {  
                    return true;  
                }  
            });  
  
            SSLContext context = SSLContext.getInstance("TLS");  
            context.init(null, new X509TrustManager[] { new X509TrustManager() {  
                public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {  
                }  
  
                public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {  
                }  
  
                public X509Certificate[] getAcceptedIssuers() {  
                    return new X509Certificate[0];  
                }  
            } }, new SecureRandom());  
            HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());  
        } catch (Exception e) {  
            // e.printStackTrace();  
        }  
    }
	
	@Override
	public void markStop() {
		running = false;
	}
}
